import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
import tensorflow as tf
from tensorflow import keras
import umap.umap_ as umap
%config InlineBackend.figure_format = 'svg'
np.random.seed(42)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 100)
data=pd.read_csv('Preprocessed_DM_xx.csv')
np.random.seed(42)
data=data.sample(frac=1) #Shuffle the data set
np.random.seed(42)
HTN_indexes=data.loc[(data['Currently.taking.a.prescribed.medicine.to.lower.BP'] != 0) | (data['First.SYSTOLIC.reading'] >= 140) | (data['First.DIASTOLIC.reading'] >= 90) ].index.values
HTN_cols=np.zeros(data.shape[0])
HTN_cols[[HTN_indexes]]=1
data['HTN']=HTN_cols
data=data.drop(["First.SYSTOLIC.reading","First.DIASTOLIC.reading","Currently.taking.a.prescribed.medicine.to.lower.BP"], axis=1)
data=data.reset_index(drop=True)
data.columns
data=data.drop(["Hb_adjust_alt_smok","Second.SYSTOLIC.reading","Second.DIASTOLIC.reading","Third.SYSTOLIC.reading","Third.DIASTOLIC.reading","Hb_status","Glucose.level",'SBP_status'], axis=1)
data=data.loc[data['BMI'] != 99.99]
data=data.loc[data['Hemoglobin.level..g.dl...1.decimal.'] != 99.99]
data=data.loc[data['Currently.has.asthma'] != .5]
data=data.loc[data['Currently.has.thyroid.disorder'] != .5]
data=data.loc[data['Currently.has.heart.disease'] != .5]
data=data.loc[data['Currently.has.cancer'] != .5]
data=data.loc[data['DM_history'] == 1]
data=data.loc[data['Type.of.caste.or.tribe.of.the.household.head'] != 0]
data=data.loc[data['Time.to.get.to.water.source..minutes.'] != -1]
data=data.drop(["Unnamed: 0","DM_status","DM_history"], axis=1)
np.random.seed(42)
i=[x for x in range(10125)]
data.set_index(pd.Series(i), inplace=True) # Reset the index
Creating 2 new dataframes: "data_disease" with features related to disease and "data_others" with rest of the features
data_disease= data[['Currently.has.asthma',
'Currently.has.thyroid.disorder', 'Currently.has.heart.disease',
'Currently.has.cancer', 'Suffers.from.TB','HTN']]
data_others= data[['Drinks.alcohol', 'Smoking_stat','Has.refrigerator',
'Has.bicycle', 'Has.motorcycle.scooter', 'Has.car.truck', 'Owns.livestock..herds.or.farm.animals','Frequency.takes.milk.or.curd',
'Frequency.eats.pulses.or.beans',
'Frequency.eats.dark.green.leafy.vegetable', 'Frequency.eats.fruits',
'Frequency.eats.eggs', 'Frequency.eats.fish',
'Frequency.eats.chicken.or.meat', 'Frequency.eats.fried.food',
'Frequency.takes.aerated.drinks','Frequency.household.members.smoke.inside.the.house','Wealth.index',
'Highest.educational.level', 'Current.age','BMI','Hemoglobin.level..g.dl...1.decimal.','Time.to.get.to.water.source..minutes.', 'Household.head.s.religion', 'Sex', 'Type.of.place.of.residence', 'Household.structure',
'Type.of.caste.or.tribe.of.the.household.head','Type.of.cooking.fuel','Source.of.drinking.water']]
def feature_clustering(UMAP_neb,min_dist_UMAP, metric, data, visual):
import umap.umap_ as umap
np.random.seed(42)
data_embedded = umap.UMAP(n_neighbors=UMAP_neb, min_dist=min_dist_UMAP, n_components=2, metric=metric, random_state=42).fit_transform(data)
data_embedded[:,0]=(data_embedded[:,0]- np.mean(data_embedded[:,0]))/np.std(data_embedded[:,0])
data_embedded[:,1]=(data_embedded[:,1]- np.mean(data_embedded[:,1]))/np.std(data_embedded[:,1])
result = pd.DataFrame(data = data_embedded ,
columns = ['UMAP_0', 'UMAP_1'])
if visual==1:
sns.lmplot( x="UMAP_0", y="UMAP_1",data=result,fit_reg=False,legend=False,scatter_kws={"s": 3},palette=customPalette_set1) # specify the point size
#plt.savefig('clusters_umap_all.png', dpi=700, bbox_inches='tight')
plt.show()
else:
pass
return result
ord_list=['Drinks.alcohol', 'Smoking_stat','Has.refrigerator',
'Has.bicycle', 'Has.motorcycle.scooter', 'Has.car.truck', 'Owns.livestock..herds.or.farm.animals','Frequency.takes.milk.or.curd',
'Frequency.eats.pulses.or.beans',
'Frequency.eats.dark.green.leafy.vegetable', 'Frequency.eats.fruits',
'Frequency.eats.eggs', 'Frequency.eats.fish',
'Frequency.eats.chicken.or.meat', 'Frequency.eats.fried.food',
'Frequency.takes.aerated.drinks','Frequency.household.members.smoke.inside.the.house','Wealth.index',
'Highest.educational.level' ]
cont_list=['Current.age','BMI','Hemoglobin.level..g.dl...1.decimal.','Time.to.get.to.water.source..minutes.']
nom_list=['Household.head.s.religion', 'Sex', 'Type.of.place.of.residence', 'Household.structure',
'Type.of.caste.or.tribe.of.the.household.head','Type.of.cooking.fuel','Source.of.drinking.water']
def FDC(data,cont_list,nom_list,ord_list,cont_metric, ord_metric, nom_metric, drop_nominal, visual):
np.random.seed(42)
colors_set1 = ["lightcoral", "lightseagreen", "mediumorchid", "orange", "burlywood", "cornflowerblue", "plum", "yellowgreen"]
customPalette_set1 = sns.set_palette(sns.color_palette(colors_set1))
cont_df=data[cont_list]
nom_df=data[nom_list]
ord_df=data[ord_list]
cont_emb=feature_clustering(30,0.1, cont_metric, cont_df, 0) #Reducing continueous features into 2dim
ord_emb=feature_clustering(30,0.1, ord_metric, ord_df, 0) #Reducing ordinal features into 2dim
nom_emb=feature_clustering(30,0.1, nom_metric, nom_df, 0) #Reducing nominal features into 2dim
if drop_nominal==1:
result_concat=pd.concat([ord_emb, cont_emb, nom_emb.drop(['UMAP_1'],axis=1)],axis=1) #concatinating all reduced dimensions to get 5D embedding(1D from nominal)
else:
result_concat=pd.concat([ord_emb, cont_emb, nom_emb],axis=1)
data_embedded = umap.UMAP(n_neighbors=30, min_dist=0.001, n_components=2, metric='euclidean', random_state=42).fit_transform(result_concat) #reducing 5D embedding to 2D using UMAP
result_reduced = pd.DataFrame(data = data_embedded ,
columns = ['UMAP_0', 'UMAP_1'])
if visual==1:
sns.lmplot( x="UMAP_0", y="UMAP_1",data=result_reduced,fit_reg=False,legend=False,scatter_kws={"s": 3},palette=customPalette_set1) # specify the point size
plt.show()
#plt.savefig('clusters_umap_all.png', dpi=700, bbox_inches='tight')
else:
pass
return result_concat, result_reduced #returns both 5D and 2D embedding
# applying Feature Distributed Clustering(FDC) on entire 10125 data with all features except disease features
entire_data_FDC_emb_five,entire_data_FDC_emb_two=FDC(data_others,cont_list,nom_list,ord_list,'euclidean','canberra','hamming',1,1)
def db_scan(eps,min_samples,five_d_embedding,two_d_embedding,visual, pal):
from sklearn.cluster import DBSCAN
dbscan = DBSCAN(eps=eps, min_samples = min_samples)
clusters=dbscan.fit_predict(five_d_embedding)
(values,counts) = np.unique(clusters,return_counts=True)
two_d_embedding['Cluster'] = clusters
if visual==1:
sns.lmplot( x="UMAP_0", y="UMAP_1",
data=two_d_embedding,
fit_reg=False,
legend=True,
hue='Cluster', # color by cluster
scatter_kws={"s": 3},palette=pal) # specify the point size
plt.savefig('dbscan_ref_5dim.png', dpi=700, bbox_inches='tight')
plt.show()
else:
pass
return two_d_embedding.Cluster.to_list(),counts
#setting color palette for visualization of clusters
colors_set1 = ['lightgray','lightcoral','cornflowerblue','orange','mediumorchid', 'lightseagreen','olive', 'chocolate','steelblue']
customPalette_set1 = sns.set_palette(sns.color_palette(colors_set1))
#Applying clustering algorithm on FDC embedding from entire data
entire_data_cluster_list,entire_data_cluster_counts=db_scan(0.8,160,entire_data_FDC_emb_five,entire_data_FDC_emb_two,1,customPalette_set1)
#Getting noise indices
non_noise_indices= np.where(np.array(entire_data_cluster_list)!=-1)
#Removing noise/outlires from FDC embedding and from entire data
entire_data_FDC_emb_five= entire_data_FDC_emb_five.iloc[non_noise_indices]
entire_data_FDC_emb_two= entire_data_FDC_emb_two.iloc[non_noise_indices]
entire_data_cluster_list= np.array(entire_data_cluster_list)[non_noise_indices]
data_others= data_others.iloc[non_noise_indices]
#Creating new cloumn for storing cluster labels
data_others['cluster_labels']= entire_data_cluster_list
#getting binary representation for cluster labels
data_others= pd.get_dummies(data=data_others, columns=['cluster_labels'])
#Getting column names of encoded cluster labels
cluster_column_names=data_others.columns[-len(np.unique(entire_data_cluster_list)):].to_list()
#75% of entire data for training
np.random.seed(42)
data=data_others.sample(frac=0.75) # Training data
#Another 25% of entire data for validation
np.random.seed(42)
data_val=data_others.drop(data.index) # Validation data
#Dividing training data into three folds
np.random.seed(42)
df_1=data.sample(frac=0.33) #fold 1
df=data.drop(df_1.index)
df_2=df.sample(frac=0.51) #fold 2
df_3=df.drop(df_2.index) #fold 3
np.random.seed(42)
#Possible combinations of concating 2 folds for training and using remaining fold for testing
training_folds=[pd.concat([df_1,df_2],axis=0), pd.concat([df_2,df_3],axis=0), pd.concat([df_3,df_1],axis=0)]
testing_folds=[df_3,df_1,df_2]
def neural_network(n_features,hidden_dim1,hidden_dim2,out_emb_size,act1,act2,loss):
np.random.seed(42)
tf.random.set_seed(42)
model=keras.Sequential([
keras.layers.Dense(hidden_dim1,input_dim=n_features,activation=act1),
keras.layers.Dense(hidden_dim1,activation=act2),
keras.layers.Dense(out_emb_size)])
model.compile(optimizer="adam" ,
loss=loss,
metrics=['mse'])
return model
def cluster_incidence_matrix_mod(cluster_list_new):
np.random.seed(42)
matrix=np.zeros((len(cluster_list_new),len(cluster_list_new)))
for i in range(len(cluster_list_new)):
for j in range(len(cluster_list_new)):
if cluster_list_new[i]==cluster_list_new[j]:
matrix[i,j]=1
else:
pass
return matrix
#Function for decoding the encoded cluster labels
def label_decoder(label_dataframe):
label_array=np.array(label_dataframe)
decoded_labels=[]
for i in label_array:
max_val=np.argmax(i)
decoded_labels.append(max_val)
return decoded_labels
colnames=[]
for i in range(len(entire_data_FDC_emb_five.columns)):
colnames.append('c'+str(i+1))
np.random.seed(42)
count=0
fold_readings=[]
while count<3:
FDC_emb_five_train=entire_data_FDC_emb_five.loc[list(training_folds[count].index)] #5D FDC embedding of training folds from entire training data
FDC_emb_two_train=entire_data_FDC_emb_two.loc[list(training_folds[count].index)] #2D embeddings of training folds from entire training data
FDC_emb_five_train.columns=colnames
#Thirty dimensional data of training fold as features_matrix(X_train)
features_matrix=np.array(training_folds[count].drop(cluster_column_names, axis=1,inplace=False)) #X_train
#Five dimensional FDC embeddings of training fold as target_matrix(y_train)
target_matrix=np.array(FDC_emb_five_train) #y_train
#Train a neural network to get five dimensional embedding
model_1=neural_network(len(features_matrix[0]),int(0.6*len(features_matrix[0])),int(0.36*len(features_matrix[0])),len(target_matrix[0]),"relu","sigmoid","mse")
history=model_1.fit(features_matrix,target_matrix,epochs=30,batch_size=8)
print('\n')
print('Training history across epochs for fold ',count+1)
plt.plot(history.history['mse'],'r')
plt.ylabel('mse')
plt.xlabel('epoch')
plt.show()
#Using same thirty dimensional features_matrix(X_train) from first neural network and encoded cluster labels of training fold as target_labels_matrix(y_train)
target_labels_matrix=np.array(training_folds[count].loc[:,cluster_column_names]) #y
#Train a neural network to get encoded cluster labels
model_2=neural_network(len(features_matrix[0]),int(0.6*len(features_matrix[0])),int(0.36*len(features_matrix[0])),len(target_labels_matrix[0]),"relu","softmax","mse")
history=model_2.fit(features_matrix,target_labels_matrix,epochs=30,batch_size=8)
print('\n')
print('Training history across epochs for fold ',count+1)
plt.plot(history.history['mse'],'r')
plt.ylabel('mse')
plt.xlabel('epoch')
plt.show()
#Decoding cluster labels of training fold
decoded_target_labels_matrix=label_decoder(target_labels_matrix)
#Actual encoded cluster labels of testing fold for metric calculation
ref_clusters=testing_folds[count].loc[:,cluster_column_names]
#Decoding encoded cluster labels of testing fold
decoded_ref_clusters=label_decoder(ref_clusters)
#predicting testing fold to get five dim embedding using trained model_1
testing_data=testing_folds[count].drop(cluster_column_names, axis=1,inplace=False)
predicted_5dim=pd.DataFrame(model_1.predict(testing_data), columns=colnames)
#UMAP on predicted 5D embedding
predicted_2dim=feature_clustering(30,0.01, "euclidean", predicted_5dim, 0)
#predicting testing fold to get encoded cluster labels using trained model_2
predicted_clusters=pd.DataFrame(model_2.predict(testing_data))
#Decoding predicted encoded cluster labels
decoded_predicted_clusters=label_decoder(predicted_clusters)
#concatinating training and predicted 5D embedding
concatenated_5dim=pd.concat([FDC_emb_five_train,predicted_5dim])
#UMAP on concatinated embedding
two_dim_viz=feature_clustering(30, 0.01, 'euclidean', concatenated_5dim, 0)
#Concatinating decoded cluster labels of training fold and predicted testing fold
concatenated_cluster_labels=np.concatenate([np.array(decoded_target_labels_matrix),np.array(decoded_predicted_clusters)+len(np.unique(decoded_target_labels_matrix))])
two_dim_viz['Cluster']= concatenated_cluster_labels
#Setting dark colors for training folds
darkerhues=['lightcoral','cornflowerblue','orange','mediumorchid', 'lightseagreen','olive', 'chocolate','steelblue']
colors_set2=[]
for i in range(len(np.unique(decoded_target_labels_matrix))):
colors_set2.append(darkerhues[i])
#Concatinating dark colors for training folds and corresponding light colors for testing folds
colors_set2=colors_set2+["lightpink", 'skyblue', 'wheat', "plum","paleturquoise", "lightgreen", 'burlywood','lightsteelblue']
print('Vizualization for FDC for training fold (shown in dark hue) '+str(count+1) + 'and predicted clusters from neural network on testing fold (shown in corresponding light hues) '+str(count+1))
#visualizing the clusters of both training and testing folds
sns.lmplot( x="UMAP_0", y="UMAP_1", data=two_dim_viz, fit_reg=False, legend=False, hue='Cluster', scatter_kws={"s": 3},palette=sns.set_palette(sns.color_palette(colors_set2)))
plt.show()
#Metric calculation
CIM_predicted=cluster_incidence_matrix_mod(np.array(decoded_predicted_clusters))#Cluster incidence metric for predicted clusters
CIM_reference=cluster_incidence_matrix_mod(np.array(decoded_ref_clusters))#Cluster incidence metric for reference clusters
Product=np.dot(CIM_predicted,CIM_reference)
cluster_incdences_in_data=np.sum(CIM_reference,axis=1)
mean_points_in_same_clusters=np.mean(np.diagonal(Product)/cluster_incdences_in_data)
fold_readings.append(mean_points_in_same_clusters*100)
print("Average percentage of patients belongs to the same cluster is: {}%".format(mean_points_in_same_clusters*100))
print('\n')
count+=1
print('\n')
print('\n')
Epoch 1/30 428/428 [==============================] - 1s 960us/step - loss: 0.6358 - mse: 0.6358 Epoch 2/30 428/428 [==============================] - 0s 975us/step - loss: 0.3503 - mse: 0.3503 Epoch 3/30 428/428 [==============================] - 0s 978us/step - loss: 0.2446 - mse: 0.2446 Epoch 4/30 428/428 [==============================] - 0s 951us/step - loss: 0.1981 - mse: 0.1981 Epoch 5/30 428/428 [==============================] - 0s 972us/step - loss: 0.1758 - mse: 0.1758 Epoch 6/30 428/428 [==============================] - 0s 964us/step - loss: 0.1607 - mse: 0.1607 Epoch 7/30 428/428 [==============================] - 0s 962us/step - loss: 0.1494 - mse: 0.1494 Epoch 8/30 428/428 [==============================] - 0s 951us/step - loss: 0.1404 - mse: 0.1404 Epoch 9/30 428/428 [==============================] - 0s 959us/step - loss: 0.1344 - mse: 0.1344 Epoch 10/30 428/428 [==============================] - 0s 945us/step - loss: 0.1290 - mse: 0.1290 Epoch 11/30 428/428 [==============================] - 0s 952us/step - loss: 0.1232 - mse: 0.1232 Epoch 12/30 428/428 [==============================] - 0s 970us/step - loss: 0.1203 - mse: 0.1203 Epoch 13/30 428/428 [==============================] - 0s 947us/step - loss: 0.1166 - mse: 0.1166 Epoch 14/30 428/428 [==============================] - 0s 936us/step - loss: 0.1133 - mse: 0.1133 Epoch 15/30 428/428 [==============================] - 0s 948us/step - loss: 0.1097 - mse: 0.1097 Epoch 16/30 428/428 [==============================] - 0s 933us/step - loss: 0.1070 - mse: 0.1070 Epoch 17/30 428/428 [==============================] - 0s 926us/step - loss: 0.1044 - mse: 0.1044 Epoch 18/30 428/428 [==============================] - 0s 915us/step - loss: 0.1027 - mse: 0.1027 Epoch 19/30 428/428 [==============================] - 0s 945us/step - loss: 0.1004 - mse: 0.1004 Epoch 20/30 428/428 [==============================] - 0s 976us/step - loss: 0.0988 - mse: 0.0988 Epoch 21/30 428/428 [==============================] - 0s 978us/step - loss: 0.0960 - mse: 0.0960 Epoch 22/30 428/428 [==============================] - 0s 955us/step - loss: 0.0945 - mse: 0.0945 Epoch 23/30 428/428 [==============================] - 0s 957us/step - loss: 0.0931 - mse: 0.0931 Epoch 24/30 428/428 [==============================] - 0s 951us/step - loss: 0.0910 - mse: 0.0910 Epoch 25/30 428/428 [==============================] - 0s 969us/step - loss: 0.0897 - mse: 0.0897 Epoch 26/30 428/428 [==============================] - 0s 962us/step - loss: 0.0886 - mse: 0.0886 Epoch 27/30 428/428 [==============================] - 0s 980us/step - loss: 0.0871 - mse: 0.0871 Epoch 28/30 428/428 [==============================] - 0s 981us/step - loss: 0.0862 - mse: 0.0862 Epoch 29/30 428/428 [==============================] - 0s 969us/step - loss: 0.0849 - mse: 0.0849 Epoch 30/30 428/428 [==============================] - 0s 959us/step - loss: 0.0841 - mse: 0.0841 Training history across epochs for fold 1
Epoch 1/30 428/428 [==============================] - 1s 964us/step - loss: 0.1554 - mse: 0.1554 Epoch 2/30 428/428 [==============================] - 0s 964us/step - loss: 0.0948 - mse: 0.0948 Epoch 3/30 428/428 [==============================] - 0s 953us/step - loss: 0.0483 - mse: 0.0483 Epoch 4/30 428/428 [==============================] - 0s 991us/step - loss: 0.0295 - mse: 0.0295 Epoch 5/30 428/428 [==============================] - 0s 951us/step - loss: 0.0223 - mse: 0.0223 Epoch 6/30 428/428 [==============================] - 0s 959us/step - loss: 0.0184 - mse: 0.0184 Epoch 7/30 428/428 [==============================] - 0s 968us/step - loss: 0.0168 - mse: 0.0168 Epoch 8/30 428/428 [==============================] - 0s 951us/step - loss: 0.0158 - mse: 0.0158 Epoch 9/30 428/428 [==============================] - 0s 963us/step - loss: 0.0142 - mse: 0.0142 Epoch 10/30 428/428 [==============================] - 0s 986us/step - loss: 0.0139 - mse: 0.0139 Epoch 11/30 428/428 [==============================] - 0s 953us/step - loss: 0.0136 - mse: 0.0136 Epoch 12/30 428/428 [==============================] - 0s 951us/step - loss: 0.0126 - mse: 0.0126 Epoch 13/30 428/428 [==============================] - 0s 961us/step - loss: 0.0120 - mse: 0.0120 Epoch 14/30 428/428 [==============================] - 0s 944us/step - loss: 0.0113 - mse: 0.0113 Epoch 15/30 428/428 [==============================] - 0s 972us/step - loss: 0.0114 - mse: 0.0114 Epoch 16/30 428/428 [==============================] - 0s 962us/step - loss: 0.0110 - mse: 0.0110 Epoch 17/30 428/428 [==============================] - 0s 967us/step - loss: 0.0109 - mse: 0.0109 Epoch 18/30 428/428 [==============================] - 0s 955us/step - loss: 0.0105 - mse: 0.0105 Epoch 19/30 428/428 [==============================] - 0s 951us/step - loss: 0.0107 - mse: 0.0107 Epoch 20/30 428/428 [==============================] - 0s 966us/step - loss: 0.0107 - mse: 0.0107 Epoch 21/30 428/428 [==============================] - 0s 962us/step - loss: 0.0100 - mse: 0.0100 Epoch 22/30 428/428 [==============================] - 0s 953us/step - loss: 0.0100 - mse: 0.0100 Epoch 23/30 428/428 [==============================] - 0s 988us/step - loss: 0.0100 - mse: 0.0100 Epoch 24/30 428/428 [==============================] - 0s 967us/step - loss: 0.0094 - mse: 0.0094 Epoch 25/30 428/428 [==============================] - 0s 952us/step - loss: 0.0094 - mse: 0.0094 Epoch 26/30 428/428 [==============================] - 0s 985us/step - loss: 0.0094 - mse: 0.0094 Epoch 27/30 428/428 [==============================] - 0s 952us/step - loss: 0.0092 - mse: 0.0092 Epoch 28/30 428/428 [==============================] - 0s 967us/step - loss: 0.0093 - mse: 0.0093 Epoch 29/30 428/428 [==============================] - 0s 981us/step - loss: 0.0088 - mse: 0.0088 Epoch 30/30 428/428 [==============================] - 0s 970us/step - loss: 0.0089 - mse: 0.0089 Training history across epochs for fold 1
53/53 [==============================] - 0s 601us/step 53/53 [==============================] - 0s 600us/step Vizualization for FDC for training fold (shown in dark hue) 1and predicted clusters from neural network on testing fold (shown in corresponding light hues) 1
Average percentage of patients belongs to the same cluster is: 92.93523126437462% Epoch 1/30 427/427 [==============================] - 1s 948us/step - loss: 0.6127 - mse: 0.6127 Epoch 2/30 427/427 [==============================] - 0s 972us/step - loss: 0.3366 - mse: 0.3366 Epoch 3/30 427/427 [==============================] - 0s 959us/step - loss: 0.2426 - mse: 0.2426 Epoch 4/30 427/427 [==============================] - 0s 962us/step - loss: 0.1976 - mse: 0.1976 Epoch 5/30 427/427 [==============================] - 0s 990us/step - loss: 0.1772 - mse: 0.1772 Epoch 6/30 427/427 [==============================] - 0s 945us/step - loss: 0.1630 - mse: 0.1630 Epoch 7/30 427/427 [==============================] - 0s 974us/step - loss: 0.1513 - mse: 0.1513 Epoch 8/30 427/427 [==============================] - 0s 991us/step - loss: 0.1433 - mse: 0.1433 Epoch 9/30 427/427 [==============================] - 0s 952us/step - loss: 0.1373 - mse: 0.1373 Epoch 10/30 427/427 [==============================] - 0s 949us/step - loss: 0.1319 - mse: 0.1319 Epoch 11/30 427/427 [==============================] - 0s 953us/step - loss: 0.1270 - mse: 0.1270 Epoch 12/30 427/427 [==============================] - 0s 962us/step - loss: 0.1230 - mse: 0.1230 Epoch 13/30 427/427 [==============================] - 0s 964us/step - loss: 0.1188 - mse: 0.1188 Epoch 14/30 427/427 [==============================] - 0s 975us/step - loss: 0.1162 - mse: 0.1162 Epoch 15/30 427/427 [==============================] - 0s 970us/step - loss: 0.1137 - mse: 0.1137 Epoch 16/30 427/427 [==============================] - 0s 972us/step - loss: 0.1094 - mse: 0.1094 Epoch 17/30 427/427 [==============================] - 0s 991us/step - loss: 0.1077 - mse: 0.1077 Epoch 18/30 427/427 [==============================] - 0s 967us/step - loss: 0.1066 - mse: 0.1066 Epoch 19/30 427/427 [==============================] - 0s 936us/step - loss: 0.1035 - mse: 0.1035 Epoch 20/30 427/427 [==============================] - 0s 970us/step - loss: 0.1014 - mse: 0.1014 Epoch 21/30 427/427 [==============================] - 0s 953us/step - loss: 0.0990 - mse: 0.0990 Epoch 22/30 427/427 [==============================] - 0s 969us/step - loss: 0.0984 - mse: 0.0984 Epoch 23/30 427/427 [==============================] - 0s 973us/step - loss: 0.0971 - mse: 0.0971 Epoch 24/30 427/427 [==============================] - 0s 953us/step - loss: 0.0943 - mse: 0.0943 Epoch 25/30 427/427 [==============================] - 0s 969us/step - loss: 0.0931 - mse: 0.0931 Epoch 26/30 427/427 [==============================] - 0s 966us/step - loss: 0.0921 - mse: 0.0921 Epoch 27/30 427/427 [==============================] - 0s 974us/step - loss: 0.0902 - mse: 0.0902 Epoch 28/30 427/427 [==============================] - 0s 977us/step - loss: 0.0903 - mse: 0.0903 Epoch 29/30 427/427 [==============================] - 0s 953us/step - loss: 0.0879 - mse: 0.0879 Epoch 30/30 427/427 [==============================] - 0s 954us/step - loss: 0.0881 - mse: 0.0881 Training history across epochs for fold 2
Epoch 1/30 427/427 [==============================] - 1s 967us/step - loss: 0.1532 - mse: 0.1532 Epoch 2/30 427/427 [==============================] - 0s 977us/step - loss: 0.0981 - mse: 0.0981 Epoch 3/30 427/427 [==============================] - 0s 982us/step - loss: 0.0493 - mse: 0.0493 Epoch 4/30 427/427 [==============================] - 0s 945us/step - loss: 0.0310 - mse: 0.0310 Epoch 5/30 427/427 [==============================] - 0s 964us/step - loss: 0.0249 - mse: 0.0249 Epoch 6/30 427/427 [==============================] - 0s 972us/step - loss: 0.0213 - mse: 0.0213 Epoch 7/30 427/427 [==============================] - 0s 955us/step - loss: 0.0192 - mse: 0.0192 Epoch 8/30 427/427 [==============================] - 0s 980us/step - loss: 0.0175 - mse: 0.0175 Epoch 9/30 427/427 [==============================] - 0s 976us/step - loss: 0.0159 - mse: 0.0159 Epoch 10/30 427/427 [==============================] - 0s 953us/step - loss: 0.0149 - mse: 0.0149 Epoch 11/30 427/427 [==============================] - 0s 985us/step - loss: 0.0143 - mse: 0.0143 Epoch 12/30 427/427 [==============================] - 0s 956us/step - loss: 0.0139 - mse: 0.0139 Epoch 13/30 427/427 [==============================] - 0s 965us/step - loss: 0.0134 - mse: 0.0134 Epoch 14/30 427/427 [==============================] - 0s 999us/step - loss: 0.0133 - mse: 0.0133 Epoch 15/30 427/427 [==============================] - 0s 953us/step - loss: 0.0129 - mse: 0.0129 Epoch 16/30 427/427 [==============================] - 0s 990us/step - loss: 0.0124 - mse: 0.0124 Epoch 17/30 427/427 [==============================] - 0s 935us/step - loss: 0.0125 - mse: 0.0125 Epoch 18/30 427/427 [==============================] - 0s 956us/step - loss: 0.0115 - mse: 0.0115 Epoch 19/30 427/427 [==============================] - 0s 963us/step - loss: 0.0117 - mse: 0.0117 Epoch 20/30 427/427 [==============================] - 0s 963us/step - loss: 0.0118 - mse: 0.0118 Epoch 21/30 427/427 [==============================] - 0s 952us/step - loss: 0.0110 - mse: 0.0110 Epoch 22/30 427/427 [==============================] - 0s 953us/step - loss: 0.0111 - mse: 0.0111 Epoch 23/30 427/427 [==============================] - 0s 992us/step - loss: 0.0108 - mse: 0.0108 Epoch 24/30 427/427 [==============================] - 0s 930us/step - loss: 0.0108 - mse: 0.0108 Epoch 25/30 427/427 [==============================] - 0s 955us/step - loss: 0.0107 - mse: 0.0107 Epoch 26/30 427/427 [==============================] - 0s 934us/step - loss: 0.0104 - mse: 0.0104 Epoch 27/30 427/427 [==============================] - 0s 976us/step - loss: 0.0101 - mse: 0.0101 Epoch 28/30 427/427 [==============================] - 0s 935us/step - loss: 0.0102 - mse: 0.0102 Epoch 29/30 427/427 [==============================] - 0s 963us/step - loss: 0.0099 - mse: 0.0099 Epoch 30/30 427/427 [==============================] - 0s 961us/step - loss: 0.0100 - mse: 0.0100 Training history across epochs for fold 2
53/53 [==============================] - 0s 601us/step 53/53 [==============================] - 0s 900us/step Vizualization for FDC for training fold (shown in dark hue) 2and predicted clusters from neural network on testing fold (shown in corresponding light hues) 2
Average percentage of patients belongs to the same cluster is: 94.48229488578839% Epoch 1/30 420/420 [==============================] - 1s 1ms/step - loss: 0.6235 - mse: 0.6235 Epoch 2/30 420/420 [==============================] - 0s 972us/step - loss: 0.3481 - mse: 0.3481 Epoch 3/30 420/420 [==============================] - 0s 973us/step - loss: 0.2440 - mse: 0.2440 Epoch 4/30 420/420 [==============================] - 0s 971us/step - loss: 0.2022 - mse: 0.2022 Epoch 5/30 420/420 [==============================] - 0s 971us/step - loss: 0.1798 - mse: 0.1798 Epoch 6/30 420/420 [==============================] - 0s 982us/step - loss: 0.1640 - mse: 0.1640 Epoch 7/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1523 - mse: 0.1523 Epoch 8/30 420/420 [==============================] - 0s 989us/step - loss: 0.1443 - mse: 0.1443 Epoch 9/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1373 - mse: 0.1373 Epoch 10/30 420/420 [==============================] - 0s 979us/step - loss: 0.1317 - mse: 0.1317 Epoch 11/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1272 - mse: 0.1272 Epoch 12/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1237 - mse: 0.1237 Epoch 13/30 420/420 [==============================] - 0s 996us/step - loss: 0.1201 - mse: 0.1201 Epoch 14/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1170 - mse: 0.1170 Epoch 15/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1139 - mse: 0.1139 Epoch 16/30 420/420 [==============================] - 0s 967us/step - loss: 0.1104 - mse: 0.1104 Epoch 17/30 420/420 [==============================] - 0s 994us/step - loss: 0.1069 - mse: 0.1069 Epoch 18/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1048 - mse: 0.1048 Epoch 19/30 420/420 [==============================] - 0s 1ms/step - loss: 0.1034 - mse: 0.1034 Epoch 20/30 420/420 [==============================] - 0s 969us/step - loss: 0.1010 - mse: 0.1010 Epoch 21/30 420/420 [==============================] - 0s 997us/step - loss: 0.0995 - mse: 0.0995 Epoch 22/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0973 - mse: 0.0973 Epoch 23/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0962 - mse: 0.0962 Epoch 24/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0942 - mse: 0.0942 Epoch 25/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0921 - mse: 0.0921 Epoch 26/30 420/420 [==============================] - 0s 988us/step - loss: 0.0916 - mse: 0.0916 Epoch 27/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0895 - mse: 0.0895 Epoch 28/30 420/420 [==============================] - 0s 984us/step - loss: 0.0884 - mse: 0.0884 Epoch 29/30 420/420 [==============================] - 0s 979us/step - loss: 0.0869 - mse: 0.0869 Epoch 30/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0862 - mse: 0.0862 Training history across epochs for fold 3
Epoch 1/30 420/420 [==============================] - 1s 975us/step - loss: 0.1529 - mse: 0.1529 Epoch 2/30 420/420 [==============================] - 0s 968us/step - loss: 0.0967 - mse: 0.0967 Epoch 3/30 420/420 [==============================] - 0s 979us/step - loss: 0.0507 - mse: 0.0507 Epoch 4/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0321 - mse: 0.0321 Epoch 5/30 420/420 [==============================] - 0s 982us/step - loss: 0.0253 - mse: 0.0253 Epoch 6/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0213 - mse: 0.0213 Epoch 7/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0188 - mse: 0.0188 Epoch 8/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0175 - mse: 0.0175 Epoch 9/30 420/420 [==============================] - 0s 984us/step - loss: 0.0163 - mse: 0.0163 Epoch 10/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0149 - mse: 0.0149 Epoch 11/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0144 - mse: 0.0144 Epoch 12/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0134 - mse: 0.0134 Epoch 13/30 420/420 [==============================] - 0s 993us/step - loss: 0.0136 - mse: 0.0136 Epoch 14/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0134 - mse: 0.0134 Epoch 15/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0125 - mse: 0.0125 Epoch 16/30 420/420 [==============================] - 0s 979us/step - loss: 0.0126 - mse: 0.0126 Epoch 17/30 420/420 [==============================] - 0s 1000us/step - loss: 0.0120 - mse: 0.0120 Epoch 18/30 420/420 [==============================] - 0s 976us/step - loss: 0.0114 - mse: 0.0114 Epoch 19/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0116 - mse: 0.0116 Epoch 20/30 420/420 [==============================] - 0s 985us/step - loss: 0.0111 - mse: 0.0111 Epoch 21/30 420/420 [==============================] - 0s 973us/step - loss: 0.0105 - mse: 0.0105 Epoch 22/30 420/420 [==============================] - 0s 975us/step - loss: 0.0105 - mse: 0.0105 Epoch 23/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0102 - mse: 0.0102 Epoch 24/30 420/420 [==============================] - 0s 990us/step - loss: 0.0102 - mse: 0.0102 Epoch 25/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0097 - mse: 0.0097 Epoch 26/30 420/420 [==============================] - 0s 989us/step - loss: 0.0099 - mse: 0.0099 Epoch 27/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0090 - mse: 0.0090 Epoch 28/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0094 - mse: 0.0094 Epoch 29/30 420/420 [==============================] - 0s 983us/step - loss: 0.0088 - mse: 0.0088 Epoch 30/30 420/420 [==============================] - 0s 1ms/step - loss: 0.0087 - mse: 0.0087 Training history across epochs for fold 3
55/55 [==============================] - 0s 579us/step 55/55 [==============================] - 0s 578us/step Vizualization for FDC for training fold (shown in dark hue) 3and predicted clusters from neural network on testing fold (shown in corresponding light hues) 3
Average percentage of patients belongs to the same cluster is: 94.86220026658046%
print('Average percentage of patients belonging to the same cluster over all three folds:', np.mean(np.array(fold_readings)))
Average percentage of patients belonging to the same cluster over all three folds: 94.09324213891449
np.random.seed(42)
FDC_emb_five_data=entire_data_FDC_emb_five.loc[list(data.index)] #5D FDC embedding of training fold from entire data
FDC_emb_two_data=entire_data_FDC_emb_two.loc[list(data.index)] #2D embedding of training fold from entire data
FDC_emb_five_data.columns=colnames
#Thirty dimensional data of training fold as features_matrix(X_train)
features_matrix=np.array(data.drop(cluster_column_names, axis=1,inplace=False)) #X_train
#Five dimensional FDC embedding of training fold as target_matrix(y_train)
target_matrix=np.array(FDC_emb_five_data) #y_train
#Train a neural network to get five dimensional embedding
model_1=neural_network(len(features_matrix[0]),int(0.6*len(features_matrix[0])),int(0.36*len(features_matrix[0])),len(target_matrix[0]),"relu","sigmoid","mse")
history=model_1.fit(features_matrix,target_matrix,epochs=30,batch_size=8)
print('\n')
print('Training history across epochs for training data ')
plt.plot(history.history['mse'],'r')
plt.ylabel('mse')
plt.xlabel('epoch')
plt.show()
#Using same thirty dimensional features_matrix(X_train) from first neural network and encoded cluster labels of training fold as target_labels_matrix(y_train)
target_labels_matrix=np.array(data.loc[:,cluster_column_names]) #y
#Train a neural network to get encoded cluster labels
model_2=neural_network(len(features_matrix[0]),int(0.6*len(features_matrix[0])),int(0.36*len(features_matrix[0])),len(target_labels_matrix[0]),"relu","softmax","mse")
history=model_2.fit(features_matrix,target_labels_matrix,epochs=30,batch_size=8)
print('\n')
print('Training history across epochs for training data ')
plt.plot(history.history['mse'],'r')
plt.ylabel('mse')
plt.xlabel('epoch')
plt.show()
#Decoding cluster labels of training fold
decoded_target_labels_matrix=label_decoder(target_labels_matrix)
#Actual encoded cluster labels of validation data for metric calculation
ref_clusters=data_val.loc[:,cluster_column_names]
#Decoding encoded cluster labels of validation data
decoded_ref_clusters=label_decoder(ref_clusters)
#predicting validation data to get five dim embedding using trained model_1
validation_data=data_val.drop(cluster_column_names, axis=1,inplace=False)
predicted_5dim=pd.DataFrame(model_1.predict(validation_data), columns=colnames)
#UMAP on predicted 5D embedding
predicted_2dim=feature_clustering(30,0.01, "euclidean", predicted_5dim, 0)
#predicting validation data to get encoded cluster labels using trained model_2
predicted_clusters=pd.DataFrame(model_2.predict(validation_data))
#Decoding predicted encoded cluster labels
decoded_predicted_clusters=label_decoder(predicted_clusters)
#concatinating training and predicted 5D embedding
concatenated_5dim=pd.concat([FDC_emb_five_data,predicted_5dim])
#UMAP on concatinated embedding
two_dim_viz=feature_clustering(30, 0.01, 'euclidean', concatenated_5dim, 0)
#Concatinating decoded cluster labels of training data and predicted validation data
concatenated_cluster_labels=np.concatenate([np.array(decoded_target_labels_matrix),np.array(decoded_predicted_clusters)+len(np.unique(decoded_target_labels_matrix))])
two_dim_viz['Cluster']= concatenated_cluster_labels
#Setting dark colors for training data
darkerhues=['lightcoral','cornflowerblue','orange','mediumorchid', 'lightseagreen','olive', 'chocolate','steelblue']
colors_set2=[]
for i in range(len(np.unique(decoded_target_labels_matrix))):
colors_set2.append(darkerhues[i])
#Concatinating dark colors for training data and corresponding light colors for validation data
colors_set2=colors_set2+["lightpink", 'skyblue', 'wheat', "plum","paleturquoise", "lightgreen", 'burlywood','lightsteelblue']
print('Vizualization for FDC for training data (shown in dark hue) '+ 'and predicted clusters from neural network on validation data (shown in corresponding light hues) ')
#visualizing the clusters of both training and validation data
sns.lmplot( x="UMAP_0", y="UMAP_1", data=two_dim_viz, fit_reg=False, legend=False, hue='Cluster', scatter_kws={"s": 3},palette=sns.set_palette(sns.color_palette(colors_set2)))
plt.show()
#Metric calculation
CIM_predicted=cluster_incidence_matrix_mod(np.array(decoded_predicted_clusters))#Cluster incidence metric for predicted clusters
CIM_reference=cluster_incidence_matrix_mod(np.array(decoded_ref_clusters))#Cluster incidence metric for reference clusters
Product=np.dot(CIM_predicted,CIM_reference)
cluster_incidences_in_data=np.sum(CIM_reference,axis=1)
mean_points_in_same_clusters=np.mean(np.diagonal(Product)/cluster_incidences_in_data)
fold_readings.append(mean_points_in_same_clusters*100)
print("Average percentage of patients belongs to the same cluster is: {}%".format(mean_points_in_same_clusters*100))
print('\n')
print('\n')
print('\n')
Epoch 1/30 638/638 [==============================] - 1s 995us/step - loss: 0.5342 - mse: 0.5342 Epoch 2/30 638/638 [==============================] - 1s 974us/step - loss: 0.2610 - mse: 0.2610 Epoch 3/30 638/638 [==============================] - 1s 985us/step - loss: 0.1955 - mse: 0.1955 Epoch 4/30 638/638 [==============================] - 1s 994us/step - loss: 0.1684 - mse: 0.1684 Epoch 5/30 638/638 [==============================] - 1s 992us/step - loss: 0.1517 - mse: 0.1517 Epoch 6/30 638/638 [==============================] - 1s 991us/step - loss: 0.1403 - mse: 0.1403 Epoch 7/30 638/638 [==============================] - 1s 996us/step - loss: 0.1321 - mse: 0.1321 Epoch 8/30 638/638 [==============================] - 1s 1ms/step - loss: 0.1272 - mse: 0.1272 Epoch 9/30 638/638 [==============================] - 1s 1ms/step - loss: 0.1224 - mse: 0.1224 Epoch 10/30 638/638 [==============================] - 1s 996us/step - loss: 0.1189 - mse: 0.1189 Epoch 11/30 638/638 [==============================] - 1s 996us/step - loss: 0.1155 - mse: 0.1155 Epoch 12/30 638/638 [==============================] - 1s 984us/step - loss: 0.1113 - mse: 0.1113 Epoch 13/30 638/638 [==============================] - 1s 998us/step - loss: 0.1077 - mse: 0.1077 Epoch 14/30 638/638 [==============================] - 1s 1ms/step - loss: 0.1050 - mse: 0.1050 Epoch 15/30 638/638 [==============================] - 1s 1ms/step - loss: 0.1014 - mse: 0.1014 Epoch 16/30 638/638 [==============================] - 1s 991us/step - loss: 0.0994 - mse: 0.0994 Epoch 17/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0962 - mse: 0.0962 Epoch 18/30 638/638 [==============================] - 1s 994us/step - loss: 0.0940 - mse: 0.0940 Epoch 19/30 638/638 [==============================] - 1s 989us/step - loss: 0.0929 - mse: 0.0929 Epoch 20/30 638/638 [==============================] - 1s 999us/step - loss: 0.0905 - mse: 0.0905 Epoch 21/30 638/638 [==============================] - 1s 983us/step - loss: 0.0886 - mse: 0.0886 Epoch 22/30 638/638 [==============================] - 1s 986us/step - loss: 0.0870 - mse: 0.0870 Epoch 23/30 638/638 [==============================] - 1s 984us/step - loss: 0.0860 - mse: 0.0860 Epoch 24/30 638/638 [==============================] - 1s 989us/step - loss: 0.0844 - mse: 0.0844 Epoch 25/30 638/638 [==============================] - 1s 995us/step - loss: 0.0837 - mse: 0.0837 Epoch 26/30 638/638 [==============================] - 1s 989us/step - loss: 0.0822 - mse: 0.0822 Epoch 27/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0812 - mse: 0.0812 Epoch 28/30 638/638 [==============================] - 1s 976us/step - loss: 0.0795 - mse: 0.0795 Epoch 29/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0792 - mse: 0.0792 Epoch 30/30 638/638 [==============================] - 1s 985us/step - loss: 0.0778 - mse: 0.0778 Training history across epochs for training data
Epoch 1/30 638/638 [==============================] - 1s 974us/step - loss: 0.1380 - mse: 0.1380 Epoch 2/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0632 - mse: 0.0632 Epoch 3/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0296 - mse: 0.0296 Epoch 4/30 638/638 [==============================] - 1s 985us/step - loss: 0.0221 - mse: 0.0221 Epoch 5/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0185 - mse: 0.0185 Epoch 6/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0164 - mse: 0.0164 Epoch 7/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0149 - mse: 0.0149 Epoch 8/30 638/638 [==============================] - 1s 995us/step - loss: 0.0136 - mse: 0.0136 Epoch 9/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0129 - mse: 0.0129 Epoch 10/30 638/638 [==============================] - 1s 996us/step - loss: 0.0125 - mse: 0.0125 Epoch 11/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0122 - mse: 0.0122 Epoch 12/30 638/638 [==============================] - 1s 986us/step - loss: 0.0117 - mse: 0.0117 Epoch 13/30 638/638 [==============================] - 1s 996us/step - loss: 0.0113 - mse: 0.0113 Epoch 14/30 638/638 [==============================] - 1s 986us/step - loss: 0.0114 - mse: 0.0114 Epoch 15/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0108 - mse: 0.0108 Epoch 16/30 638/638 [==============================] - 1s 2ms/step - loss: 0.0106 - mse: 0.0106 Epoch 17/30 638/638 [==============================] - 1s 2ms/step - loss: 0.0104 - mse: 0.0104 Epoch 18/30 638/638 [==============================] - 1s 2ms/step - loss: 0.0098 - mse: 0.0098 Epoch 19/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0097 - mse: 0.0097 Epoch 20/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0096 - mse: 0.0096 Epoch 21/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0097 - mse: 0.0097 Epoch 22/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0096 - mse: 0.0096 Epoch 23/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0096 - mse: 0.0096 Epoch 24/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0094 - mse: 0.0094 Epoch 25/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0093 - mse: 0.0093 Epoch 26/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0091 - mse: 0.0091 Epoch 27/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0093 - mse: 0.0093 Epoch 28/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0093 - mse: 0.0093 Epoch 29/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0085 - mse: 0.0085 Epoch 30/30 638/638 [==============================] - 1s 1ms/step - loss: 0.0088 - mse: 0.0088 Training history across epochs for training data
54/54 [==============================] - 0s 884us/step 54/54 [==============================] - 0s 884us/step Vizualization for FDC for training data (shown in dark hue) and predicted clusters from neural network on validation data (shown in corresponding light hues)
Average percentage of patients belongs to the same cluster is: 94.81152691395197%